/**
 * AnalyzerUtils.java	1.0 2007 六月 13 17:17:58,创建
 */

package lia.ch4;

import java.io.IOException;
import java.io.StringReader;
import java.util.ArrayList;
import java.util.List;

import org.apache.commons.logging.Log;
import org.apache.commons.logging.LogFactory;
import org.apache.lucene.analysis.Analyzer;
import org.apache.lucene.analysis.Token;
import org.apache.lucene.analysis.TokenStream;

/**
 *
 * @author 杨帆
 * @version 1.0 2007 六月 13 17:17:58,创建
 */
public class AnalyzerUtils
{
	@SuppressWarnings("unused")
	private static final Log log = LogFactory.getLog(AnalyzerUtils.class);

	public static void displayTokens(Analyzer analyzer, String text) throws IOException
	{
		final List<Token> tokenList = tokensFromAnalysis(analyzer, text);

		int pos = 0;
		for(Token token : tokenList)
		{
			int increament = token.getPositionIncrement();
			if(increament > 0)
			{
				pos += increament;
			}
			log.debug(pos + ": " + tokenToString(token));
		}
	}

	public static String tokenToString(Token token)
	{
		return "[" + token.termText() + ":" + token.startOffset() + "->" + token.endOffset() + ":" + token.type() + "]";
	}

	private static List<Token> tokensFromAnalysis(Analyzer analyzer, String text) throws IOException
	{
		TokenStream stream = analyzer.tokenStream("contents", new StringReader(text));
		List<Token> tokenList = new ArrayList<Token>();
		while(true)
		{
			Token token = stream.next();
			if(token == null) break;
			tokenList.add(token);
		}
		return tokenList;
	}
}
